//
// Created by 11510 on 2024/7/28.
//

#ifndef DICTIONARY_DICTIONARYPRODUCER_H
#define DICTIONARY_DICTIONARYPRODUCER_H

#include "cppjieba/Jieba.hpp"
#include <string>
#include <vector>
#include <map>
#include <set>
#include <iostream>
#include <dirent.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fstream>
#include <sstream>
#include <algorithm>
#include <cctype>
#include <memory>

using std::vector;
using std::string;
using std::pair;
using std::map;
using std::set;
using std::cerr;
using std::endl;
using std::ifstream;
using std::ofstream;
using std::istringstream;

class DictionaryProducer {
private:
    vector<string> files; /* 语料文件的绝对路径集合 */

    set<string> stopWord;   /* 停用词 */
    map<string, int> wordMap;    /* 词典，转换前 */
    vector<pair<string, int>> dictionary; /* 词典,转换后 */

    map<string, set<int>> index;  /* 词典索引 */

    cppjieba::Jieba *cuttor;


public:

    ~DictionaryProducer() = default;

    DictionaryProducer(const string &directory, cppjieba::Jieba *cuttor) :
            cuttor(cuttor) {
        getAllFiles(directory);
    }

//    创建词典
    void buildDict() {
        for (const auto &path: files) {
            ifstream ifs;
            ifs.open(path);
            if (!ifs) {
                cerr << "ifstream open file failed" << endl;
                return;
            }

            string line;

            /* 一行一行处理 */
            while (getline(ifs, line)) {
                /*删标点，大写转小*/
                cleanString(line);
                std::vector<std::string> words;
                cuttor->Cut(line, words, true);

                for (const auto &word: words) {
                    if (!stopWord.count(word) && isChineseOrEnglish(word)) {
                        wordMap[word]++;
                    }
                }
            }

            ifs.close();
        }
    }

//    创建词典vector,索引
    void createIndex() {
        /* 存入词典, 索引 */
        int lineNumber = 0; /* 行号 */
        for (map<string, int>::iterator it = wordMap.begin(); it != wordMap.end(); it++, lineNumber++) {
            dictionary.push_back({it->first, it->second});

            std::vector<std::string> single;
            if (isWord(it->first)) {
                single = std::move(splitWordToLetters(it->first));
            }else{
                single = std::move(splitChineseCharacters(it->first));
            }

            for (auto &elem: single) {
                index[elem].insert(lineNumber);
            }
        }
    }

//    写入文件
    void storeDict(const string &filepath) {
        ofstream ofs(filepath);
        if (!ofs) {
            cerr << "ofstream open file failed" << endl;
            return;
        }

        std::for_each (dictionary.begin(),dictionary.end(),[&ofs](const std::pair<std::string,int>& pair){
            ofs << pair.first << " " << pair.second << "\n";
        });

        ofs.close();
    }
    void storeIndex(const string &filepath) {
        ofstream ofs(filepath);
        if (!ofs) {
            cerr << "ofstream open file failed" << endl;
            return;
        }

        for (const auto &elem: index) {
            ofs << elem.first << " ";
            for (const auto &entry: elem.second) {
                ofs << entry << " ";
            }
            ofs << "\n";
        }
        ofs.close();
    }

//    创建停用词
    void setStopWord(const string &filePath) {
        std::ifstream file(filePath);
        if (!file.is_open()) {
            std::cerr << "无法打开文件: " << filePath << std::endl;
            return;
        }

        std::string word;
        while (std::getline(file, word)) {
            stopWord.insert(word);
        }

        file.close();
    }

private:
    void getAllFiles(const string &directory) {
        DIR *dir;
        struct dirent *entry;
        struct stat info;

        if ((dir = opendir(directory.c_str())) != nullptr) {
            while ((entry = readdir(dir)) != nullptr) {
                std::string path = directory + "/" + entry->d_name;

                if (entry->d_type == DT_DIR) {
                    // 忽略 "." 和 ".."
                    if (std::string(entry->d_name) == "." || std::string(entry->d_name) == "..")
                        continue;
                    // 递归处理子目录
                    getAllFiles(path);
                } else {
                    // 检查是否为常规文件
                    if (stat(path.c_str(), &info) == 0 && S_ISREG(info.st_mode)) {
                        // 获取绝对路径
                        char real_path[PATH_MAX];
                        realpath(path.c_str(), real_path);
                        files.push_back(std::string(real_path));
                    }
                }
            }
            closedir(dir);
        } else {
            std::cerr << "无法打开目录: " << directory << std::endl;
        }
    }

    void cleanString(std::string &str) {
        // 删除标点符号,连词符
        str.erase(std::remove_if(str.begin(), str.end(), [](unsigned char c) {
            return std::ispunct(c) || c == '-';
        }), str.end());

        // 将大写字母转换为小写
        std::transform(str.begin(), str.end(), str.begin(), [](unsigned char c) {
            return std::tolower(c);
        });
    }

    // 判断字符串是否是单词
    bool isWord(const std::string &str) {
        // 空字符串不是一个单词
        if (str.empty()) {
            return false;
        }
        // 遍历字符串中的每个字符，检查是否都是字母
        for (char c: str) {
            if (!std::isalpha(static_cast<unsigned char>(c))) {
                return false;
            }
        }
        return true;
    }

    // 将单词分割为字母并返回一个包含这些字母的向量
    std::vector<std::string> splitWordToLetters(const std::string &word) {
        std::vector<std::string> letters;
        for (char c: word) {
            letters.push_back(std::string(1, c));
        }
        return letters;
    }

    // 将中文词组分割成汉字返回一个包含该汉字的向量
    vector<std::string> splitChineseCharacters(const std::string &str) {
        auto result = std::vector<std::string>();
        for (size_t i = 0; i < str.size();) {
            unsigned char c = str[i];
            int len = 0;

            // Determine the length of the UTF-8 character
            if ((c & 0x80) == 0) {
                len = 1; // ASCII
            } else if ((c & 0xE0) == 0xC0) {
                len = 2; // 2-byte UTF-8
            } else if ((c & 0xF0) == 0xE0) {
                len = 3; // 3-byte UTF-8
            } else if ((c & 0xF8) == 0xF0) {
                len = 4; // 4-byte UTF-8
            } else {
                // Invalid UTF-8 character
                std::cerr << "Invalid UTF-8 encoding at position " << i << std::endl;
                break;
            }

            result.emplace_back(str.substr(i, len));
            i += len;
        }
        return result;
    }

    // 判断字符是否是英文字符
    bool isEnglishChar(char c) {
        return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');
    }

    // 判断字符是否是中文字符（UTF-8编码）
    bool isChineseChar(const std::string &str, size_t &index) {
        unsigned char c1 = str[index];
        if (c1 >= 0xE4 && c1 <= 0xE9) {
            if (index + 2 < str.size()) {
                unsigned char c2 = str[index + 1];
                unsigned char c3 = str[index + 2];
                // 检查是否是合法的UTF-8编码
                if ((c2 & 0xC0) == 0x80 && (c3 & 0xC0) == 0x80) {
                    index += 3; // 中文字符占3个字节
                    return true;
                }
            }
        }
        return false;
    }

    // 判断字符串是否是中文汉字或英文字母
    bool isChineseOrEnglish(const std::string &str) {
        size_t i = 0;
        while (i < str.size()) {
            if ((unsigned char) str[i] <= 0x7F) { // 单字节字符（包括ASCII字符）
                if (!isEnglishChar(str[i])) {
                    return false;
                }
                ++i;
            } else { // 多字节字符（可能是中文字符）
                if (!isChineseChar(str, i)) {
                    return false;
                }
            }
        }
        return true;
    }

};


#endif //DICTIONARY_DICTIONARYPRODUCER_H
